import jieba
import pandas as pd

from __00__config import Config


def data_preprocessing(datapath, preprocessed_datapath, is_char=True):
	# 读取原数据文件
	df = pd.read_csv(datapath, encoding='utf-8', sep=',')
	# 查看数据信息
	# print(df.info())
	# print(df.head())
	# 对数据进行处理
	# 将问题文本拿出来
	with open(preprocessed_datapath, 'w', encoding='utf-8') as fw:
		# 获取问题文本和标签
		questions = [line.strip() for line in df['questions']]
		labels = [line for line in df['labels']]
		for idx, (question, label) in enumerate(zip(questions, labels)):
			# 将标签转为对应的文本
			label_str = config.id2class_dict[label]
			# print(label_str)
			# fasttext数据的标签文本格式为：__label__label_name
			# 判断是需要分词保存还是分字符保存
			if is_char:
				text_split = ' '.join(list(question))
			else:
				text_split = ' '.join(jieba.lcut(question))
			# 拼接成fasttext数据格式
			fn_line = '__label__' + label_str + ' ' + text_split + '\n'
			fw.write(fn_line)


if __name__ == '__main__':
	config = Config()
	data_preprocessing(config.train_datapath, config.processed_char_train_datapath, True)
	data_preprocessing(config.test_datapath, config.processed_char_test_datapath, True)
	data_preprocessing(config.dev_datapath, config.processed_char_dev_datapath, True)
	data_preprocessing(config.train_datapath, config.processed_word_train_datapath, False)
	data_preprocessing(config.test_datapath, config.processed_word_test_datapath, False)
	data_preprocessing(config.dev_datapath, config.processed_word_dev_datapath, False)
